<html>
 <head>
  <meta charset="utf-8"/>
  <meta content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no" name="viewport"/>
  <title>
   【投稿】Machine Learning With Spark Note 2:构建简单的推荐系统  | 数螺 | NAUT IDEA
  </title>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap-theme.min.css" rel="stylesheet"/>
  <link href="http://cdn.bootcss.com/bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet"/>
  <style type="text/css">
   #xmain img {
                  max-width: 100%;
                  display: block;
                  margin-top: 10px;
                  margin-bottom: 10px;
                }

                #xmain p {
                    line-height:150%;
                    font-size: 16px;
                    margin-top: 20px;
                }

                #xmain h2 {
                    font-size: 24px;
                }

                #xmain h3 {
                    font-size: 20px;
                }

                #xmain h4 {
                    font-size: 18px;
                }


                .header {
	           background-color: #0099ff;
	           color: #ffffff;
	           margin-bottom: 20px;
	        }

	        .header p {
                  margin: 0px;
                  padding: 10px 0;
                  display: inline-block;  
                  vertical-align: middle;
                  font-size: 16px;
               }

               .header a {
                 color: white;
               }

              .header img {
                 height: 25px;
              }
  </style>
  <script src="http://cdn.bootcss.com/jquery/3.0.0/jquery.min.js">
  </script>
  <script src="http://nautstatic-10007657.file.myqcloud.com/static/css/readability.min.js" type="text/javascript">
  </script>
  <script type="text/javascript">
   $(document).ready(function() {
                 var loc = document.location;
                 var uri = {
                  spec: "http://dataunion.org/22536.html",
                  host: "http://dataunion.org",
                  prePath: "http://dataunion.org",
                  scheme: "http",
                  pathBase: "http://dataunion.org/"
                 };
    
                 var documentClone = document.cloneNode(true);
                 var article = new Readability(uri, documentClone).parse();
     
                 document.getElementById("xmain").innerHTML = article.content;
                });
  </script>
  <!-- 1466460552: Accept with keywords: (title(0.4):推荐系统,投稿,社区,Spark,数盟, topn(0.533333333333):社区,数盟,信息,行业资讯,推荐系统,协同,Python,模型,计算,文章,物品,Spark,排序,数据,向量,投稿,场景,函数,python,矩阵,用户,建模,算法,影片,相似性,spark,产品,编程语言,代码,模块).-->
 </head>
 <body onload="">
  <div class="header">
   <div class="container">
    <div class="row">
     <div class="col-xs-6 col-sm-6 text-left">
      <a href="/databee">
       <img src="http://nautidea-10007657.cos.myqcloud.com/logo_white.png"/>
      </a>
      <a href="/databee">
       <p>
        数螺
       </p>
      </a>
     </div>
     <div class="hidden-xs col-sm-6 text-right">
      <p>
       致力于数据科学的推广和知识传播
      </p>
     </div>
    </div>
   </div>
  </div>
  <div class="container text-center">
   <h1>
    【投稿】Machine Learning With Spark Note 2:构建简单的推荐系统
   </h1>
  </div>
  <div class="container" id="xmain">
   ﻿﻿
   <title>
    【投稿】Machine Learning With Spark Note 2:构建简单的推荐系统 | 数盟社区
   </title>
   <!-- All in One SEO Pack 2.2.7.6.2 by Michael Torbert of Semper Fi Web Design[32,95] -->
   <!-- /all in one seo pack -->
   <!--
<div align="center">
<a href="http://strata.oreilly.com.cn/hadoop-big-data-cn?cmp=mp-data-confreg-home-stcn16_dataunion_pc" target="_blank"><img src="http://dataunion.org/wp-content/uploads/2016/05/stratabj.jpg"/ ></a>
</div>
-->
   <header id="header-web">
    <div class="header-main">
     <hgroup class="logo">
      <h1>
       <a href="http://dataunion.org/" rel="home" title="数盟社区">
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/logo.png"/>
       </a>
      </h1>
     </hgroup>
     <!--logo-->
     <nav class="header-nav">
      <ul class="menu" id="menu-%e4%b8%bb%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-71" id="menu-item-71">
        <a href="http://dataunion.org/category/events" title="events">
         活动
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22457" id="menu-item-22457">
          <a href="http://dataunion.org/2016timeline">
           2016档期
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22459" id="menu-item-22459">
          <a href="http://dataunion.org/category/parterc">
           合作会议
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor menu-item-has-children menu-item-20869" id="menu-item-20869">
        <a href="http://dataunion.org/category/tech" title="articles">
         文章
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20867" id="menu-item-20867">
          <a href="http://dataunion.org/category/tech/base" title="base">
           基础架构
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3302" id="menu-item-3302">
          <a href="http://dataunion.org/category/tech/ai" title="ai">
           人工智能
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3303" id="menu-item-3303">
          <a href="http://dataunion.org/category/tech/analysis" title="analysis">
           数据分析
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21920" id="menu-item-21920">
          <a href="http://dataunion.org/category/tech/dm">
           数据挖掘
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-3314" id="menu-item-3314">
          <a href="http://dataunion.org/category/tech/viz" title="viz">
           可视化
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-3305" id="menu-item-3305">
          <a href="http://dataunion.org/category/tech/devl" title="devl">
           编程语言
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-20876" id="menu-item-20876">
        <a href="http://dataunion.org/category/industry">
         行业
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-16328" id="menu-item-16328">
          <a href="http://dataunion.org/category/industry/case" title="case">
           行业应用
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-2112" id="menu-item-2112">
          <a href="http://dataunion.org/category/industry/demo" title="demo">
           Demo展示
          </a>
         </li>
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21562" id="menu-item-21562">
          <a href="http://dataunion.org/category/industry/news">
           行业资讯
          </a>
         </li>
        </ul>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category current-post-ancestor current-menu-parent current-post-parent menu-item-311" id="menu-item-311">
        <a href="http://dataunion.org/category/sources" title="sources">
         资源
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20870" id="menu-item-20870">
        <a href="http://dataunion.org/category/books" title="book">
         图书
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-21363" id="menu-item-21363">
        <a href="http://dataunion.org/category/training">
         课程
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-has-children menu-item-21853" id="menu-item-21853">
        <a href="http://dataunion.org/category/jobs">
         职位
        </a>
        <ul class="sub-menu">
         <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-22050" id="menu-item-22050">
          <a href="http://dataunion.org/category/career">
           职业规划
          </a>
         </li>
        </ul>
       </li>
      </ul>
     </nav>
     <!--header-nav-->
    </div>
   </header>
   <!--header-web-->
   <div id="main">
    <div id="soutab">
     <form action="http://dataunion.org/" class="search" method="get">
     </form>
    </div>
    <div id="container">
     <nav id="mbx">
      当前位置：
      <a href="http://dataunion.org">
       首页
      </a>
      &gt;
      <a href="http://dataunion.org/category/sources">
       资源
      </a>
      &gt;  正文
     </nav>
     <!--mbx-->
     <article class="content">
      <header align="centre" class="contenttitle">
       <div class="mscc">
        <h1 class="mscctitle">
         <a href="http://dataunion.org/22536.html">
          【投稿】Machine Learning With Spark Note 2:构建简单的推荐系统
         </a>
        </h1>
        <address class="msccaddress ">
         <em>
          1,781 次阅读 -
         </em>
         <a href="http://dataunion.org/category/sources" rel="category tag">
          资源
         </a>
         ,
         <a href="http://dataunion.org/category/tech/devl" rel="category tag">
          编程语言
         </a>
        </address>
       </div>
      </header>
      <div class="content-text">
       <p>
        本文为数盟特约作者投稿，欢迎转载，请注明出处“数盟社区”和作者
       </p>
       <p>
        博主简介：段石石，1号店精准化推荐算法工程师，主要负责1号店用户画像构建，喜欢钻研点Machine Learning的黑科技，对Deep Learning感兴趣，喜欢玩kaggle、看9神，对数据和Machine Learning有兴趣咱们可以一起聊聊，个人博客：
        <a href="http://hacker.duanshishi.com/" target="_blank">
         hacker.duanshishi.com
        </a>
       </p>
       <h3>
        推荐引擎应用场景：
       </h3>
       <ul>
        <li>
         用户有海量选择：随着场景内item越来越多，用户越来越难以选择到合适的产品
        </li>
        <li>
         个性化场景：在选择产品时，会借鉴那些与推荐用户相似地群体，利用群体智慧对用户进行推荐”千人千面”
        </li>
       </ul>
       <p>
        在本篇博客中，会涉及到以下几个部分：
       </p>
       <ul>
        <li>
         介绍不同类型的推荐引擎
        </li>
        <li>
         使用用户偏好模型来构造推荐模型
        </li>
        <li>
         使用训练好的模型来为指定user计算给定item的相似度大的items
        </li>
        <li>
         使用标准的评测函数来构造推荐模型的好坏
        </li>
       </ul>
       <p>
       </p>
       <h3>
        推荐模型类别：
       </h3>
       <ul>
        <li>
         基于item的过滤：使用item的内容或者属性，选择给定item的相似的item列表，这些属性一般为文本内容，包括题目、名、标签以及一些产品的元信息，通常也包括一些media信息，比如图像、音频等等
        </li>
        <li>
         协同过滤：协同过滤是一种集体智慧的推荐模型，在基于用户的协同过滤方法中，如果两个用户有相似的偏好（通过用户对物品的评分、用户查看物品的记录、用户对物品的评论），当为给定用户来推荐相关产品时，会使用其他相似偏好的用户的产品列表来对该用户进行推荐。基于item的协同过滤，一般数据组成为用户和用户对某些items的rating，产品被相似偏好的用户rating相同的趋势比较大，因而我们可以用所有用户对物品的偏好，来发现物品与物品之间的相似度，根据用户的历史偏好物品，根据相似信息来推荐给该用户
        </li>
        <li>
         Matrix Factorization
        </li>
       </ul>
       <p>
        因为在Spark的MLlib模块中只有MF算法，文章之后会讲述如何使用Matrix Factorization来做相关的推荐。
       </p>
       <h3>
        Matrix Factorization
       </h3>
       <p>
        MF在Netflix Prize中得到最好的名词，关于MF的一片overview：
        <a href="http://techblog.netflix.com/2012/04/netflix-recommendations-beyond-5-stars.html">
         http://techblog.netflix.com/2012/04/netflix-recommendations-beyond-5-stars.html
        </a>
        。
       </p>
       <h4>
        Explicit matrix factorization
       </h4>
       <p>
        user ratings 数据：
       </p>
       <div class="page" title="Page 101">
        <div class="layoutArea">
         <div class="column">
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-57686985660ef222091344" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             Tom, Star Wars, 5
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-57686985660ef222091344-1">
                  1
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-57686985660ef222091344-1">
                  <span class="crayon-v">
                   Tom
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   Star
                  </span>
                  <span class="crayon-v">
                   Wars
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-cn">
                   5
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0012 seconds] -->
          <br/>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566100401803661" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             Jane, Titanic, 4
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566100401803661-1">
                  1
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566100401803661-1">
                  <span class="crayon-v">
                   Jane
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   Titanic
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-cn">
                   4
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0006 seconds] -->
          <br/>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566107807512736" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             Bill, Batman, 3
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566107807512736-1">
                  1
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566107807512736-1">
                  <span class="crayon-v">
                   Bill
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   Batman
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-cn">
                   3
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0006 seconds] -->
          <br/>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-576869856610d298247992" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             Jane, Star Wars, 2
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-576869856610d298247992-1">
                  1
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-576869856610d298247992-1">
                  <span class="crayon-v">
                   Jane
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   Star
                  </span>
                  <span class="crayon-v">
                   Wars
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-cn">
                   2
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0007 seconds] -->
          <br/>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566113703212591" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             Bill, Titanic, 3
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566113703212591-1">
                  1
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566113703212591-1">
                  <span class="crayon-v">
                   Bill
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   Titanic
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-cn">
                   3
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0006 seconds] -->
          <br/>
          以user为行，movie为列构造对应rating matrix：
          <p>
          </p>
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage68.png"/>
          </p>
          <p>
           MF就是一种直接建模user-item矩阵的方法，利用两个低维度的小矩阵的乘积来表示，属于一种降维的技术。
          </p>
          <p>
           如果我们有U个用户，I个items，若不经过MF处理，它看来会使这样的：
          </p>
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage70.png"/>
          </p>
          <p>
           是一个极其稀疏的矩阵，经过MF处理后，表示为两个维度较小的矩阵相乘：
          </p>
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage72.png"/>
          </p>
          <p>
           这类模型被称为latent feature models，旨在寻找那些潜在的特征，来间接表示user-item rating的矩阵。这类潜在的features并不直接建模user对item的rating关系，而是通过latent features更趋近于建模用户对某类items的偏好，例如某类影片、风格等等，而这些事通过MF寻找其内在的信息，无需items的详细描述（和基于content的方法不同）。
          </p>
          <p>
           MF模型如何计算一个user对某个item的偏好，对应向量相乘即可：
          </p>
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage74.png"/>
          </p>
          <p>
           如何计算两个item的相似度：
          </p>
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage76.png"/>
          </p>
          <p>
           MF模型的好处是一旦模型创建好后，predict变得十分容易，并且性能也很好，但是在海量的用户和itemset时，存储和生产MF中的如上图的这两个矩阵会变得具有挑战性。
          </p>
          <h3>
           Implicit matrix factorization
          </h3>
          <p>
           前面我们都在讨论显式的一些偏好信息，比如rating，但是在大部分应用中，拿不到这类信息，我们更多滴搜集的是一些隐性的反馈信息，这类反馈信息没有明确地告诉某个用户对某个item的偏好信息，但是却可以从用户对某个item的交互信息中建模出来，例如一些二值特征，包括是否浏览过、是否购买过产品、以及多少次看过某部电影等等。
          </p>
          <p>
           MLlib中提供了一种处理这类隐性特征的方法，将前面的输入ratings矩阵其实可以看做是两个矩阵：二值偏好矩阵P和信心权重矩阵C；
          </p>
          <p>
           举个例子：假定我们的网站上面没有设计对movie的rating部分，只能通过log查看到用户是否观看过影片，然后通过后期处理，可以看出他观看到过多少次某部影片，这里P来表示影片是否被某用户看过，C来描述这里的confidence weighting也就是观看的次数：
          </p>
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage78.png"/>
          </p>
          <p>
           这里我们把P和C的dot product来替代前面的rating矩阵，那么我们最终建模来预估某用户对item的偏好
          </p>
          <h3>
           Alternating least squares
          </h3>
          <p>
           ALS是解决MF问题的一个优化技术，被证明高效、高性能并且能有效地并行化，目前为止，是MLlib中推荐模块的唯一一个算法。Spark官网上有
           <a href="http://spark.apache.org/docs/latest/mllib-collaborative-filtering.html">
            专门地描述
           </a>
           。
          </p>
          <h3>
           特征提取
          </h3>
          <p>
           特征提取是从已有数据中找到有用的数据来对算法进行建模，本文中使用显式数据也就是用户对movie的rating信息，这个数据来源于网络上的MovieLens标准数据集，以下代码为《Machine Learning with Spark》这本书里面的python的重写版本，会有专门的ipython notebook放到github上。
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-576869856611c574814142" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             rawData = sc.textFile("../data/ML_spark/MovieLens/u.data")
print rawData.first()
rawRatings = rawData.map(lambda x: x.split('\t'))
rawRatings.take(5)
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-576869856611c574814142-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-576869856611c574814142-2">
                  2
                 </div>
                 <div class="crayon-num" data-line="crayon-576869856611c574814142-3">
                  3
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-576869856611c574814142-4">
                  4
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-576869856611c574814142-1">
                  <span class="crayon-v">
                   rawData
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   sc
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   textFile
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-s">
                   "../data/ML_spark/MovieLens/u.data"
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-576869856611c574814142-2">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   rawData
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   first
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-576869856611c574814142-3">
                  <span class="crayon-v">
                   rawRatings
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   rawData
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   split
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-s">
                   '\t'
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-576869856611c574814142-4">
                  <span class="crayon-v">
                   rawRatings
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   take
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-cn">
                   5
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0021 seconds] -->
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage80.png"/>
          </p>
          <p class="”prettyprint”">
           数据分别是userId，itemId，rating和timestamp。
          </p>
          <p>
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566123199412342" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             from pyspark.mllib.recommendation import Rating
from pyspark.mllib.recommendation import ALS
ratings = rawRatings.map(lambda x : Rating(int(x[0]),int(x[1]),float(x[2])))
print ratings.first()
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566123199412342-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566123199412342-2">
                  2
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566123199412342-3">
                  3
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566123199412342-4">
                  4
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566123199412342-1">
                  <span class="crayon-st">
                   from
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   pyspark
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   mllib
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   recommendation
                  </span>
                  <span class="crayon-r">
                   import
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   Rating
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566123199412342-2">
                  <span class="crayon-st">
                   from
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   pyspark
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   mllib
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   recommendation
                  </span>
                  <span class="crayon-r">
                   import
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   ALS
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566123199412342-3">
                  <span class="crayon-v">
                   ratings
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   rawRatings
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   Rating
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-k ">
                   int
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-cn">
                   0
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-k ">
                   int
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-cn">
                   1
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-k ">
                   float
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-cn">
                   2
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566123199412342-4">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   ratings
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   first
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0034 seconds] -->
          <p>
           格式化数据，用于后面建模数据，导入Rating，ALS模块，下面是ALS类的使用说明：
          </p>
          <p class="”prettyprint”">
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage81.png"/>
          </p>
          <p class="”prettyprint”">
           其中rank就是上面latent feature model中矩阵的k，在下面的实验中，我们设为50：
          </p>
          <p>
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-576869856612a238933777" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             model = ALS.train(ratings,50)
# modelImplicit = ALS.(ratings,50,alpha=0.02)
userFeatures = model.userFeatures()
print userFeatures.take(2)
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-576869856612a238933777-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-576869856612a238933777-2">
                  2
                 </div>
                 <div class="crayon-num" data-line="crayon-576869856612a238933777-3">
                  3
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-576869856612a238933777-4">
                  4
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-576869856612a238933777-1">
                  <span class="crayon-v">
                   model
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   ALS
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   train
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   ratings
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-cn">
                   50
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-576869856612a238933777-2">
                  <span class="crayon-c">
                   # modelImplicit = ALS.(ratings,50,alpha=0.02)
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-576869856612a238933777-3">
                  <span class="crayon-v">
                   userFeatures
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   model
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   userFeatures
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-576869856612a238933777-4">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   userFeatures
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   take
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-cn">
                   2
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0017 seconds] -->
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage82.png"/>
          </p>
          <p>
           这里user1与user2，均用50维的向量来表示，也就是上面U*k那个矩阵的每个向量
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566130642275923" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             predictRating = model.predict(789,123)
print predictRating
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566130642275923-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566130642275923-2">
                  2
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566130642275923-1">
                  <span class="crayon-v">
                   predictRating
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   model
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   predict
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-cn">
                   789
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-cn">
                   123
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566130642275923-2">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   predictRating
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0010 seconds] -->
          <p>
           预测用户789对item 123的rating值，结果为3.76599662082。
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566135735742098" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             topKRecs = model.recommendProducts(userId,K)
for rec in topKRecs:
    print rec
moviesForUser = ratings.groupBy(lambda x : x.user).mapValues(list).lookup(userId)
# print moviesForUser
for i in sorted(moviesForUser[0],key=lambda x : x.rating,reverse=True):
    print i.product
# for
# print moviesForUser
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566135735742098-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566135735742098-2">
                  2
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566135735742098-3">
                  3
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566135735742098-4">
                  4
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566135735742098-5">
                  5
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566135735742098-6">
                  6
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566135735742098-7">
                  7
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566135735742098-8">
                  8
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566135735742098-9">
                  9
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566135735742098-1">
                  <span class="crayon-v">
                   topKRecs
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   model
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   recommendProducts
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   K
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566135735742098-2">
                  <span class="crayon-st">
                   for
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   rec
                  </span>
                  <span class="crayon-st">
                   in
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   topKRecs
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566135735742098-3">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   rec
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566135735742098-4">
                  <span class="crayon-v">
                   moviesForUser
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   ratings
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   groupBy
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   user
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   mapValues
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-k ">
                   list
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   lookup
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566135735742098-5">
                  <span class="crayon-c">
                   # print moviesForUser
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566135735742098-6">
                  <span class="crayon-st">
                   for
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-i">
                   i
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   in
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-k ">
                   sorted
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   moviesForUser
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-cn">
                   0
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   key
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   reverse
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-t">
                   True
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566135735742098-7">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   i
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   product
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566135735742098-8">
                  <span class="crayon-c">
                   # for
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566135735742098-9">
                  <span class="crayon-c">
                   # print moviesForUser
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0042 seconds] -->
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage87.png"/>
          </p>
          <p>
           使用recommendProducts来为用户推荐top10的items，其items顺序为降序。MoviesForUser是从ratings数据中找出的用户789rating最高的数据，仔细看下发现数据和我们的ratings里面找出的数据貌似一个都没有相同的，那么是不是说明我们的算法不给力呢？！这个可不一定，想想看，如果推荐系统只是推荐给你看过的电影，那么它一定是一个失败的，并且完全对系统的kpi数据无提升作用，前面提到，MF的实质是通过latent feature去找到与用户过去偏好高的有某些隐性相同特征的电影（这些由整体用户的集体智慧得到），比如可能是某一类型的电影、又或者相同的演员等等，所以这里不能说明推荐系统不给力，但是确实也很难具有解释性。
          </p>
          <h3>
           Item recommendations
          </h3>
          <p>
           基于MF的方法中，我们可以利用之前看到k*I的矩阵，计算两个向量质检的相似性，也就是item的相似性。这样，可以很容易做相似商品推荐的场景。这里我们定义相似函数为余弦相似性：
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-576869856613d571862334" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             import numpy as np
def cosineSImilarity(x,y):
    return np.dot(x,y)/(np.linalg.norm(x)*np.linalg.norm(y))
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-576869856613d571862334-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-576869856613d571862334-2">
                  2
                 </div>
                 <div class="crayon-num" data-line="crayon-576869856613d571862334-3">
                  3
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-576869856613d571862334-1">
                  <span class="crayon-r">
                   import
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   numpy
                  </span>
                  <span class="crayon-st">
                   as
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   np
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-576869856613d571862334-2">
                  <span class="crayon-r">
                   def
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   cosineSImilarity
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   y
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-576869856613d571862334-3">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   return
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   np
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   dot
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   y
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   /
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   np
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   linalg
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   norm
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   *
                  </span>
                  <span class="crayon-v">
                   np
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   linalg
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   norm
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   y
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0023 seconds] -->
          <p>
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566142883728576" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             testx = np.array([1.0,2.0,3.0])
print cosineSImilarity(testx,testx)
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566142883728576-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566142883728576-2">
                  2
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566142883728576-1">
                  <span class="crayon-v">
                   testx
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   np
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   array
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-cn">
                   1.0
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-cn">
                   2.0
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-cn">
                   3.0
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566142883728576-2">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   cosineSImilarity
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   testx
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   testx
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0014 seconds] -->
          <p>
           然后，通过ALS建模的item的向量，拿到对应地item的向量表示：
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566148192972851" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             itemId = 567
itemFactor = model.productFeatures().lookup(itemId)[0]
# itemFactor = itemFactor[1]
print itemFactor
# model.productFeatures().collect()
sims = model.productFeatures().map(lambda (id,factor):(id,cosineSImilarity(np.array(factor),
             np.array(itemFactor))))
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566148192972851-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566148192972851-2">
                  2
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566148192972851-3">
                  3
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566148192972851-4">
                  4
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566148192972851-5">
                  5
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566148192972851-6">
                  6
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566148192972851-7">
                  7
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566148192972851-1">
                  <span class="crayon-v">
                   itemId
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-cn">
                   567
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566148192972851-2">
                  <span class="crayon-v">
                   itemFactor
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   model
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   productFeatures
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   lookup
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   itemId
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-cn">
                   0
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566148192972851-3">
                  <span class="crayon-c">
                   # itemFactor = itemFactor[1]
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566148192972851-4">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   itemFactor
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566148192972851-5">
                  <span class="crayon-c">
                   # model.productFeatures().collect()
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566148192972851-6">
                  <span class="crayon-v">
                   sims
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   model
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   productFeatures
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-k ">
                   id
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   factor
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-k ">
                   id
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-e">
                   cosineSImilarity
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   np
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   array
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   factor
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566148192972851-7">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   np
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   array
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   itemFactor
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0032 seconds] -->
          <p>
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-576869856614e121485000" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             sims.sortBy(lambda (x,y):y,ascending=False).take(10)
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-576869856614e121485000-1">
                  1
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-576869856614e121485000-1">
                  <span class="crayon-v">
                   sims
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   sortBy
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   y
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-v">
                   y
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   ascending
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-t">
                   False
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   take
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-cn">
                   10
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0013 seconds] -->
          <p>
           利用ALS的item向量拿到itemId为567的向量表示，然后对model的item的特征向量来计算与567的相似度，按降序排序并取top10
          </p>
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage85.png"/>
          </p>
          <p>
           这样，可以找到与567这个item相似性最大的itemlist。
          </p>
          <h3>
           如何衡量推荐系统的性能
          </h3>
          <p>
           怎么判断我们生成的模型性能呢？常用的有一些比如Mean Squared Error，Root Mean Squared Error，但是这类标准无法考量推荐最终的items的排序问题，在实际工作中用的比较多的是Mean Average Precision，考虑到了item的排序造成的影响。
          </p>
          <p>
           MSE&amp;RMSE：
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566155622588350" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             userProducts = ratings.map(lambda rating:(rating.user,rating.product))
print userProducts.take(1)[0]
predictions = model.predictAll(userProducts).map(lambda rating:((rating.user,rating.product)
              ,rating.rating))
print predictions.take(5)
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566155622588350-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566155622588350-2">
                  2
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566155622588350-3">
                  3
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566155622588350-4">
                  4
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566155622588350-5">
                  5
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566155622588350-1">
                  <span class="crayon-v">
                   userProducts
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   ratings
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   user
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   product
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566155622588350-2">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   userProducts
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   take
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-cn">
                   1
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-cn">
                   0
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566155622588350-3">
                  <span class="crayon-v">
                   predictions
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   model
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   predictAll
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userProducts
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   user
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   product
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566155622588350-4">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566155622588350-5">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   predictions
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   take
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-cn">
                   5
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0037 seconds] -->
          <p>
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-576869856615b382697877" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             ratingsAndPredictions = ratings.map(lambda rating:((rating.user,rating.product),rating.rating))
             .join(predictions)
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-576869856615b382697877-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-576869856615b382697877-2">
                  2
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-576869856615b382697877-1">
                  <span class="crayon-v">
                   ratingsAndPredictions
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   ratings
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   user
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   product
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-576869856615b382697877-2">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   join
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   predictions
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0019 seconds] -->
          <p>
           MSE = ratingsAndPredictions.map(lambda ((x,y),(m,n)):math.pow(m-n,2)).reduce(lambda x,y:x+y)/ratingsAndPredictions.count() print MSE print math.sqrt(MSE)
          </p>
          <p>
           先map ratings数据得到用户对item的组合，然后对这类数据predictAll计算该用户对item的rating估计值。然后利用join函数将预测的数据与ratings中的数据”联合”起来，塞入相似度函数进行计算,最终结果如下：
          </p>
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage86.png"/>
          </p>
          <p>
           备注：看到这里肯定有人会问题，你之前在前面recommendProducts的，没有一个item是与ratings的数据相同，但是这里为什么又对比ratings中的评分信息来衡量推荐模型的好坏呢。猜想：recommendProduct是基于最终预测的ratings的高低来推荐的，但是，考虑到前面分析的原因，应该是不仅仅是按predict的rating的高低来给定推荐产品而是参入了其他的考量，所以这里并不矛盾。
          </p>
          <p>
           APK：
          </p>
          <p>
           什么是APK？可以看下
           <a href="https://github.com/benhamner/Metrics">
            这里
           </a>
           ，里面有R，Matlab，Python的各种Metrics的实现，还有
           <a href="https://www.kaggle.com/wiki/MeanAveragePrecision">
            kaggle
           </a>
           里对APK的说明，逻辑很简单，相对于MSE和RMSE，考虑了推荐的排序对最后metrics的影响，如果检索出来的item排序越靠前，得分越高。
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566162961699796" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             def avgPrecisionK(actual, predicted,k=10):
    if len(predicted)&gt;k:
        predicted = predicted[:k]

    score = 0.0
    num_hits = 0.0
    for i,p in enumerate(predicted):
        if p in actual and p not in predicted[:i]:
            num_hits += 1.0
            score += num_hits / (i+1.0)

    if not actual:
        return 1.0

    return score / min(len(actual), k)
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566162961699796-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566162961699796-2">
                  2
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566162961699796-3">
                  3
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566162961699796-4">
                  4
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566162961699796-5">
                  5
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566162961699796-6">
                  6
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566162961699796-7">
                  7
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566162961699796-8">
                  8
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566162961699796-9">
                  9
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566162961699796-10">
                  10
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566162961699796-11">
                  11
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566162961699796-12">
                  12
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566162961699796-13">
                  13
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566162961699796-14">
                  14
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566162961699796-15">
                  15
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566162961699796-1">
                  <span class="crayon-r">
                   def
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   avgPrecisionK
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   actual
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   predicted
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   k
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-cn">
                   10
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566162961699796-2">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   if
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-k ">
                   len
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   predicted
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   &gt;
                  </span>
                  <span class="crayon-v">
                   k
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566162961699796-3">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   predicted
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   predicted
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-v">
                   k
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566162961699796-4">
                 </div>
                 <div class="crayon-line" id="crayon-5768698566162961699796-5">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   score
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-cn">
                   0.0
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566162961699796-6">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   num_hits
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-cn">
                   0.0
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566162961699796-7">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   for
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   i
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-i">
                   p
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   in
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-k ">
                   enumerate
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   predicted
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566162961699796-8">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   if
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-i">
                   p
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   in
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   actual
                  </span>
                  <span class="crayon-st">
                   and
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-i">
                   p
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   not
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   in
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   predicted
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-v">
                   i
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566162961699796-9">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   num_hits
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   +=
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-cn">
                   1.0
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566162961699796-10">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   score
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   +=
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   num_hits
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   /
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   i
                  </span>
                  <span class="crayon-o">
                   +
                  </span>
                  <span class="crayon-cn">
                   1.0
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566162961699796-11">
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566162961699796-12">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   if
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   not
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   actual
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566162961699796-13">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   return
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-cn">
                   1.0
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566162961699796-14">
                 </div>
                 <div class="crayon-line" id="crayon-5768698566162961699796-15">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   return
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   score
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   /
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-k ">
                   min
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-k ">
                   len
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   actual
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   k
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0062 seconds] -->
          <p>
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566168912880437" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             itemFactors = model.productFeatures().map(lambda (id,factor):factor).collect()
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566168912880437-1">
                  1
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566168912880437-1">
                  <span class="crayon-v">
                   itemFactors
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   model
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   productFeatures
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-k ">
                   id
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   factor
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-v">
                   factor
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   collect
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0015 seconds] -->
          <p>
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-576869856616e593042409" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             itemMatrix = np.array(itemFactors)
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-576869856616e593042409-1">
                  1
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-576869856616e593042409-1">
                  <span class="crayon-v">
                   itemMatrix
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   np
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   array
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   itemFactors
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0008 seconds] -->
          <p>
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566174807440358" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             imBroadcast = sc.broadcast(itemMatrix)
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566174807440358-1">
                  1
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566174807440358-1">
                  <span class="crayon-v">
                   imBroadcast
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   sc
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   broadcast
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   itemMatrix
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0010 seconds] -->
          <p>
           拿到product的所有向量表示，初始化矩阵 ，然后broadcast到各个节点。
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566179668217427" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             userVector = model.userFeatures().map(lambda (userId,array):(userId,np.array(array)))
# print userVector[0]
userVector = userVector.map(lambda (userId,x): 
            (userId,imBroadcast.value.dot((np.array(x).transpose()))))
userVectorId = userVector.map(lambda (userId,x) : (userId,[(xx,i) for i,xx in enumerate(x.tolist())]))
sortUserVectorId = userVectorId.map(lambda (userId,x):(userId,sorted(x,key=lambda x:x[0],reverse=True)))
sortUserVectorRecId = sortUserVectorId.map(lambda (userId,x): (userId,[xx[1] for xx in x]))
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566179668217427-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566179668217427-2">
                  2
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566179668217427-3">
                  3
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566179668217427-4">
                  4
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566179668217427-5">
                  5
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566179668217427-6">
                  6
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566179668217427-7">
                  7
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566179668217427-1">
                  <span class="crayon-v">
                   userVector
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   model
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   userFeatures
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-k ">
                   array
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   np
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   array
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-k ">
                   array
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566179668217427-2">
                  <span class="crayon-c">
                   # print userVector[0]
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566179668217427-3">
                  <span class="crayon-v">
                   userVector
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   userVector
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-h">
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566179668217427-4">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   imBroadcast
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   value
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   dot
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   np
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   array
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   transpose
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566179668217427-5">
                  <span class="crayon-v">
                   userVectorId
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   userVector
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   xx
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   i
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   for
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   i
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-e">
                   xx
                  </span>
                  <span class="crayon-st">
                   in
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-k ">
                   enumerate
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   tolist
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566179668217427-6">
                  <span class="crayon-v">
                   sortUserVectorId
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   userVectorId
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-k ">
                   sorted
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   key
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-cn">
                   0
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   reverse
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-t">
                   True
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566179668217427-7">
                  <span class="crayon-v">
                   sortUserVectorRecId
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   sortUserVectorId
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-v">
                   xx
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-cn">
                   1
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   for
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   xx
                  </span>
                  <span class="crayon-st">
                   in
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0090 seconds] -->
          <p>
           为每一个user推荐一个对应的item list，并按user向量与item向量相乘计算的该用户对该item的rating值来进行排序，最终给定一个有序的item的list。
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566180960138038" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             userMovies = ratings.map(lambda rating: (rating.user,rating.product)).groupBy(lambda (x,y):x)
userMovies = userMovies.map(lambda (userId,x):(userId, [xx[1] for xx in x] ))
allAPK=sortUserVectorRecId.join(userMovies).map(lambda (userId,(predicted, actual))
       :avgPrecisionK(actual,predicted,2000))
print allAPK.reduce(lambda x,y:x+y)/allAPK.count()
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566180960138038-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566180960138038-2">
                  2
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566180960138038-3">
                  3
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566180960138038-4">
                  4
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566180960138038-5">
                  5
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566180960138038-1">
                  <span class="crayon-v">
                   userMovies
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   ratings
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   user
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   rating
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   product
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   groupBy
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   y
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566180960138038-2">
                  <span class="crayon-v">
                   userMovies
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   userMovies
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-v">
                   xx
                  </span>
                  <span class="crayon-sy">
                   [
                  </span>
                  <span class="crayon-cn">
                   1
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-st">
                   for
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   xx
                  </span>
                  <span class="crayon-st">
                   in
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   ]
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566180960138038-3">
                  <span class="crayon-v">
                   allAPK
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-v">
                   sortUserVectorRecId
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   join
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userMovies
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   predicted
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   actual
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566180960138038-4">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-e">
                   avgPrecisionK
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   actual
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   predicted
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-cn">
                   2000
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566180960138038-5">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   allAPK
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   reduce
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   y
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-v">
                   x
                  </span>
                  <span class="crayon-o">
                   +
                  </span>
                  <span class="crayon-v">
                   y
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-o">
                   /
                  </span>
                  <span class="crayon-v">
                   allAPK
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   count
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0060 seconds] -->
          <p>
           然后从rating中找到对应的的item 列表，然后塞入之前我们写的apk函数，然后求平均，最终结果为0.115484271925。
          </p>
          <p>
           当然我们可以直接使用MLlib内置的evaluation模块来对我们的模型进行评价，如MSE，RMSE：
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-5768698566186568577394" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             from pyspark.mllib.evaluation import RegressionMetrics
from pyspark.mllib.evaluation import RankingMetrics
predictedAndTrue = ratingsAndPredictions.map(lambda ((userId,product),(predicted, actual))
      :(predicted,actual))
# print predictedAndTrue.take(1)
regressionMetrics = RegressionMetrics(predictedAndTrue)
print "Mean Squared Error = %f"%regressionMetrics.meanSquaredError
print "Root Mean Squared Error %f"% regressionMetrics.rootMeanSquaredError
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-5768698566186568577394-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566186568577394-2">
                  2
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566186568577394-3">
                  3
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566186568577394-4">
                  4
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566186568577394-5">
                  5
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566186568577394-6">
                  6
                 </div>
                 <div class="crayon-num" data-line="crayon-5768698566186568577394-7">
                  7
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-5768698566186568577394-8">
                  8
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-5768698566186568577394-1">
                  <span class="crayon-st">
                   from
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   pyspark
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   mllib
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   evaluation
                  </span>
                  <span class="crayon-r">
                   import
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   RegressionMetrics
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566186568577394-2">
                  <span class="crayon-st">
                   from
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   pyspark
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   mllib
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   evaluation
                  </span>
                  <span class="crayon-r">
                   import
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   RankingMetrics
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566186568577394-3">
                  <span class="crayon-v">
                   predictedAndTrue
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   ratingsAndPredictions
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   product
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   predicted
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   actual
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566186568577394-4">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   predicted
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   actual
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566186568577394-5">
                  <span class="crayon-c">
                   # print predictedAndTrue.take(1)
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566186568577394-6">
                  <span class="crayon-v">
                   regressionMetrics
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   RegressionMetrics
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   predictedAndTrue
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-5768698566186568577394-7">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-s">
                   "Mean Squared Error = %f"
                  </span>
                  <span class="crayon-o">
                   %
                  </span>
                  <span class="crayon-v">
                   regressionMetrics
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   meanSquaredError
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-5768698566186568577394-8">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-s">
                   "Root Mean Squared Error %f"
                  </span>
                  <span class="crayon-o">
                   %
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   regressionMetrics
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-v">
                   rootMeanSquaredError
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0038 seconds] -->
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage88.png"/>
          </p>
          <p>
           MAP：
          </p>
          <!-- Crayon Syntax Highlighter v_2.7.2_beta -->
          <div class="crayon-syntax crayon-theme-classic crayon-font-monaco crayon-os-pc print-yes notranslate" data-settings=" minimize scroll-mouseover" id="crayon-576869856618c456779962" style=" margin-top: 12px; margin-bottom: 12px; font-size: 12px !important; line-height: 15px !important;">
           <div class="crayon-toolbar" data-settings=" mouseover overlay hide delay" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
            <span class="crayon-title">
            </span>
            <div class="crayon-tools" style="font-size: 12px !important;height: 18px !important; line-height: 18px !important;">
             <div class="crayon-button crayon-nums-button" title="切换是否显示行编号">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-plain-button" title="纯文本显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-wrap-button" title="切换自动换行">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-expand-button" title="点击展开代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-copy-button" title="复制代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <div class="crayon-button crayon-popup-button" title="在新窗口中显示代码">
              <div class="crayon-button-icon">
              </div>
             </div>
             <span class="crayon-language">
              Python
             </span>
            </div>
           </div>
           <div class="crayon-info" style="min-height: 16.8px !important; line-height: 16.8px !important;">
           </div>
           <div class="crayon-plain-wrap">
            <textarea class="crayon-plain print-no" data-settings="dblclick" readonly="" style="-moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4; font-size: 12px !important; line-height: 15px !important;" wrap="soft">
             #MAP
# The implementation of the average precision at the K function in RankingMetrics is slightly different 
# from ours,
# so we will get different results. However, the computation of the overall mean average precision 
#(MAP, which does not use a threshold at K) is the same as our function if we select K to be very high 
# (say, at least as high as the number of items in our item set)
sortedLabels = sortUserVectorRecId.join(userMovies).map(lambda (userId,(predicted, actual))
             :(predicted,actual))
# print sortedLabels.take(1)
rankMetrics = RankingMetrics(sortedLabels)
print "Mean Average Precision = %f" % rankMetrics.meanAveragePrecision
print "Mean Average Precision(at K=10) = %f" % rankMetrics.precisionAt(5)
            </textarea>
           </div>
           <div class="crayon-main" style="">
            <table class="crayon-table">
             <tbody>
              <tr class="crayon-row">
               <td class="crayon-nums " data-settings="show">
                <div class="crayon-nums-content" style="font-size: 12px !important; line-height: 15px !important;">
                 <div class="crayon-num" data-line="crayon-576869856618c456779962-1">
                  1
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-576869856618c456779962-2">
                  2
                 </div>
                 <div class="crayon-num" data-line="crayon-576869856618c456779962-3">
                  3
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-576869856618c456779962-4">
                  4
                 </div>
                 <div class="crayon-num" data-line="crayon-576869856618c456779962-5">
                  5
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-576869856618c456779962-6">
                  6
                 </div>
                 <div class="crayon-num" data-line="crayon-576869856618c456779962-7">
                  7
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-576869856618c456779962-8">
                  8
                 </div>
                 <div class="crayon-num" data-line="crayon-576869856618c456779962-9">
                  9
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-576869856618c456779962-10">
                  10
                 </div>
                 <div class="crayon-num" data-line="crayon-576869856618c456779962-11">
                  11
                 </div>
                 <div class="crayon-num crayon-striped-num" data-line="crayon-576869856618c456779962-12">
                  12
                 </div>
                </div>
               </td>
               <td class="crayon-code">
                <div class="crayon-pre" style="font-size: 12px !important; line-height: 15px !important; -moz-tab-size:4; -o-tab-size:4; -webkit-tab-size:4; tab-size:4;">
                 <div class="crayon-line" id="crayon-576869856618c456779962-1">
                  <span class="crayon-c">
                   #MAP
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-576869856618c456779962-2">
                  <span class="crayon-c">
                   # The implementation of the average precision at the K function in RankingMetrics is slightly different
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-576869856618c456779962-3">
                  <span class="crayon-c">
                   # from ours,
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-576869856618c456779962-4">
                  <span class="crayon-c">
                   # so we will get different results. However, the computation of the overall mean average precision
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-576869856618c456779962-5">
                  <span class="crayon-c">
                   #(MAP, which does not use a threshold at K) is the same as our function if we select K to be very high
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-576869856618c456779962-6">
                  <span class="crayon-c">
                   # (say, at least as high as the number of items in our item set)
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-576869856618c456779962-7">
                  <span class="crayon-v">
                   sortedLabels
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   sortUserVectorRecId
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   join
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userMovies
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-k ">
                   map
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-r">
                   lambda
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   userId
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   predicted
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   actual
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-576869856618c456779962-8">
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   :
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   predicted
                  </span>
                  <span class="crayon-sy">
                   ,
                  </span>
                  <span class="crayon-v">
                   actual
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-576869856618c456779962-9">
                  <span class="crayon-c">
                   # print sortedLabels.take(1)
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-576869856618c456779962-10">
                  <span class="crayon-v">
                   rankMetrics
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   =
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-e">
                   RankingMetrics
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-v">
                   sortedLabels
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                 <div class="crayon-line" id="crayon-576869856618c456779962-11">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-s">
                   "Mean Average Precision = %f"
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   %
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   rankMetrics
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   meanAveragePrecision
                  </span>
                 </div>
                 <div class="crayon-line crayon-striped-line" id="crayon-576869856618c456779962-12">
                  <span class="crayon-k ">
                   print
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-s">
                   "Mean Average Precision(at K=10) = %f"
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-o">
                   %
                  </span>
                  <span class="crayon-h">
                  </span>
                  <span class="crayon-v">
                   rankMetrics
                  </span>
                  <span class="crayon-sy">
                   .
                  </span>
                  <span class="crayon-e">
                   precisionAt
                  </span>
                  <span class="crayon-sy">
                   (
                  </span>
                  <span class="crayon-cn">
                   5
                  </span>
                  <span class="crayon-sy">
                   )
                  </span>
                 </div>
                </div>
               </td>
              </tr>
             </tbody>
            </table>
           </div>
          </div>
          <!-- [Format Time: 0.0037 seconds] -->
          <p>
           <img src="http://blogburness1-wordpress.stor.sinaapp.com/uploads/2015/12/NewImage89.png"/>
          </p>
          <p>
           这里结果与我们前面取k=2000的结果相同，说明我们的计算和MLlib是一致的，但是K=10或者比较小的值时，不一样，这是因为MLlib在precisionAt(k)这个函数与我们前面逻辑不同，这里我们不做考虑。
          </p>
          <p>
           本章的代码放到了
           <a href="https://github.com/burness/ML_Spark">
            github上面
           </a>
           ，是ipython notebook的可以直接调用试用下，这版代码是我学习spark写的，水平很差，而且notebook中也没有基本的代码说明，算是对原书中这部分的scala的一次重写，喜欢python和spark的可以研究下，一步一步看下还是会熟悉python操作spark的流程的
          </p>
         </div>
        </div>
       </div>
      </div>
      <div>
       <strong>
        注：转载文章均来自于公开网络，仅供学习使用，不会用于任何商业用途，如果侵犯到原作者的权益，请您与我们联系删除或者授权事宜，联系邮箱：contact@dataunion.org。转载数盟网站文章请注明原文章作者，否则产生的任何版权纠纷与数盟无关。
       </strong>
      </div>
      <!--content_text-->
      <div class="fenxian">
       <!-- JiaThis Button BEGIN -->
       <div class="jiathis_style_32x32">
        <p class="jiathis_button_weixin">
        </p>
        <p class="jiathis_button_tsina">
        </p>
        <p class="jiathis_button_qzone">
        </p>
        <p class="jiathis_button_cqq">
        </p>
        <p class="jiathis_button_tumblr">
        </p>
        <a class="jiathis jiathis_txt jtico jtico_jiathis" href="http://www.jiathis.com/share" target="_blank">
        </a>
        <p class="jiathis_counter_style">
        </p>
       </div>
       <!-- JiaThis Button END -->
      </div>
     </article>
     <!--content-->
     <!--相关文章-->
     <div class="xianguan">
      <div class="xianguantitle">
       相关文章！
      </div>
      <ul class="pic">
       <li>
        <a href="http://dataunion.org/24678.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/06/20140917125452915416-216x200.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/24678.html" rel="bookmark" title="python3中的正则模块">
         python3中的正则模块
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/24675.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/06/t015b337bd75d9ef893-161x200.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/24675.html" rel="bookmark" title="注释是恶魔，请不要再写一行注释">
         注释是恶魔，请不要再写一行注释
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/24660.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/06/u16130037972892789947fm21gp0-300x157.jpg"/>
        </a>
        <a class="link" href="http://dataunion.org/24660.html" rel="bookmark" title="如何从Github上轻松安装R包">
         如何从Github上轻松安装R包
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/24654.html">
         <img src="http://dataunion.org/wp-content/uploads/2016/06/df53fac99fc53ba5a90666abcca25e6d_b-267x200.png"/>
        </a>
        <a class="link" href="http://dataunion.org/24654.html" rel="bookmark" title="简单形象又有趣地说说强大的神经网络">
         简单形象又有趣地说说强大的神经网络
        </a>
       </li>
      </ul>
     </div>
     <!--相关文章-->
     <div class="comment" id="comments">
      <!-- You can start editing here. -->
      <!-- If comments are open, but there are no comments. -->
      <div class="title">
       期待你一针见血的评论，Come on！
      </div>
      <div id="respond">
       <p>
        不用想啦，马上
        <a href="http://dataunion.org/wp-login.php?redirect_to=http%3A%2F%2Fdataunion.org%2F22536.html">
         "登录"
        </a>
        发表自已的想法.
       </p>
      </div>
     </div>
     <!-- .nav-single -->
    </div>
    <!--Container End-->
    <aside id="sitebar">
     <div class="sitebar_list2">
      <div class="wptag">
       <span class="tagtitle">
        热门标签+
       </span>
       <div class="tagg">
        <ul class="menu" id="menu-%e5%8f%8b%e6%83%85%e9%93%be%e6%8e%a5">
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-1605" id="menu-item-1605">
          <a href="http://taidizh.com/">
           泰迪智慧
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20884" id="menu-item-20884">
          <a href="http://www.transwarp.cn/">
           星环科技
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-3538" id="menu-item-3538">
          <a href="http://datall.org/">
           珈和遥感
          </a>
         </li>
         <li class="menu-item menu-item-type-custom menu-item-object-custom menu-item-20888" id="menu-item-20888">
          <a href="http://www.chinahadoop.cn/">
           小象学院
          </a>
         </li>
        </ul>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <div class="textwidget">
       <div align="center">
        <a href="http://study.163.com/course/courseMain.htm?courseId=991022" target="_blank">
         <img src="http://dataunion.org/wp-content/uploads/2016/03/dv.jpg"/>
        </a>
       </div>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       文章分类
      </h4>
      <div class="tagcloud">
       <a class="tag-link-44" href="http://dataunion.org/category/industry/demo" style="font-size: 10.204724409449pt;" title="4个话题">
        Demo展示
       </a>
       <a class="tag-link-31" href="http://dataunion.org/category/experts" style="font-size: 15.826771653543pt;" title="52个话题">
        专家团队
       </a>
       <a class="tag-link-870" href="http://dataunion.org/category/tech/ai" style="font-size: 19.795275590551pt;" title="273个话题">
        人工智能
       </a>
       <a class="tag-link-488" href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f" style="font-size: 8pt;" title="1个话题">
        加入数盟
       </a>
       <a class="tag-link-869" href="http://dataunion.org/category/tech/viz" style="font-size: 17.204724409449pt;" title="93个话题">
        可视化
       </a>
       <a class="tag-link-30" href="http://dataunion.org/category/partners" style="font-size: 10.645669291339pt;" title="5个话题">
        合作伙伴
       </a>
       <a class="tag-link-889" href="http://dataunion.org/category/parterc" style="font-size: 11.582677165354pt;" title="8个话题">
        合作会议
       </a>
       <a class="tag-link-104" href="http://dataunion.org/category/books" style="font-size: 12.96062992126pt;" title="15个话题">
        图书
       </a>
       <a class="tag-link-220" href="http://dataunion.org/category/tech/base" style="font-size: 19.850393700787pt;" title="281个话题">
        基础架构
       </a>
       <a class="tag-link-219" href="http://dataunion.org/category/tech/analysis" style="font-size: 19.409448818898pt;" title="232个话题">
        数据分析
       </a>
       <a class="tag-link-887" href="http://dataunion.org/category/tech/dm" style="font-size: 13.291338582677pt;" title="17个话题">
        数据挖掘
       </a>
       <a class="tag-link-34" href="http://dataunion.org/category/tech" style="font-size: 20.732283464567pt;" title="404个话题">
        文章
       </a>
       <a class="tag-link-1" href="http://dataunion.org/category/uncategorized" style="font-size: 22pt;" title="693个话题">
        未分类
       </a>
       <a class="tag-link-4" href="http://dataunion.org/category/events" style="font-size: 14.503937007874pt;" title="29个话题">
        活动
       </a>
       <a class="tag-link-890" href="http://dataunion.org/category/tech/%e6%b7%b1%e5%ba%a6%e5%ad%a6%e4%b9%a0" style="font-size: 10.204724409449pt;" title="4个话题">
        深度学习
       </a>
       <a class="tag-link-221" href="http://dataunion.org/category/tech/devl" style="font-size: 18.968503937008pt;" title="193个话题">
        编程语言
       </a>
       <a class="tag-link-888" href="http://dataunion.org/category/career" style="font-size: 15.661417322835pt;" title="48个话题">
        职业规划
       </a>
       <a class="tag-link-5" href="http://dataunion.org/category/jobs" style="font-size: 14.11811023622pt;" title="25个话题">
        职位
       </a>
       <a class="tag-link-871" href="http://dataunion.org/category/industry" style="font-size: 15.716535433071pt;" title="49个话题">
        行业
       </a>
       <a class="tag-link-613" href="http://dataunion.org/category/industry/case" style="font-size: 16.984251968504pt;" title="84个话题">
        行业应用
       </a>
       <a class="tag-link-885" href="http://dataunion.org/category/industry/news" style="font-size: 17.425196850394pt;" title="102个话题">
        行业资讯
       </a>
       <a class="tag-link-10" href="http://dataunion.org/category/training" style="font-size: 14.228346456693pt;" title="26个话题">
        课程
       </a>
       <a class="tag-link-16" href="http://dataunion.org/category/sources" style="font-size: 15.661417322835pt;" title="48个话题">
        资源
       </a>
      </div>
     </div>
     <div class="sitebar_list">
      <h4 class="sitebar_title">
       功能
      </h4>
      <ul>
       <li>
        <a href="http://dataunion.org/wp-login.php?action=register">
         注册
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/wp-login.php">
         登录
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/feed">
         文章
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="http://dataunion.org/comments/feed">
         评论
         <abbr title="Really Simple Syndication">
          RSS
         </abbr>
        </a>
       </li>
       <li>
        <a href="https://cn.wordpress.org/" title="基于WordPress，一个优美、先进的个人信息发布平台。">
         WordPress.org
        </a>
       </li>
      </ul>
     </div>
    </aside>
    <div class="clear">
    </div>
   </div>
   <!--main-->
   ﻿
   <footer id="dibu">
    <div class="about">
     <div class="right">
      <ul class="menu" id="menu-%e5%ba%95%e9%83%a8%e8%8f%9c%e5%8d%95">
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-18024" id="menu-item-18024">
        <a href="http://dataunion.org/category/partners">
         合作伙伴
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20881" id="menu-item-20881">
        <a href="http://dataunion.org/contribute">
         文章投稿
        </a>
       </li>
       <li class="menu-item menu-item-type-taxonomy menu-item-object-category menu-item-20872" id="menu-item-20872">
        <a href="http://dataunion.org/category/%e5%8a%a0%e5%85%a5%e6%95%b0%e7%9b%9f">
         加入数盟
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-22441" id="menu-item-22441">
        <a href="http://dataunion.org/f-links">
         友情链接
        </a>
       </li>
       <li class="menu-item menu-item-type-post_type menu-item-object-page menu-item-20874" id="menu-item-20874">
        <a href="http://dataunion.org/aboutus">
         关于数盟
        </a>
       </li>
      </ul>
      <p class="banquan">
       数盟社区        ，
        做最棒的数据科学社区
      </p>
     </div>
     <div class="left">
      <ul class="bottomlist">
       <li>
        <a href="http://weibo.com/DataScientistUnion  " target="_blank" 　title="">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weibo.png"/>
        </a>
       </li>
       <li>
        <a class="cd-popup-trigger" href="http://dataunion.org/22536.html#0">
         <img src="http://dataunion.org/wp-content/themes/yzipi/images/weixin.png"/>
        </a>
       </li>
      </ul>
      <div class="cd-popup">
       <div class="cd-popup-container">
        <h1>
         扫描二维码,加微信公众号
        </h1>
        <img src="http://dataunion.org/wp-content/themes/yzipi/images/2014-12-06-1515289049.png"/>
        <a class="cd-popup-close" href="http://dataunion.org/22536.html">
        </a>
       </div>
       <!-- cd-popup-container -->
      </div>
      <!-- cd-popup -->
     </div>
    </div>
    <!--about-->
    <div class="bottom">
     <a href="http://dataunion.org/">
      数盟社区
     </a>
     <a href="http://www.miitbeian.gov.cn/" rel="external nofollow" target="_blank">
      京ICP备14026740号
     </a>
     联系我们：
     <a href="mailto:contact@dataunion.org" target="_blank">
      contact@dataunion.org
     </a>
     <div class="tongji">
     </div>
     <!--bottom-->
     <div class="scroll" id="scroll" style="display:none;">
      ︿
     </div>
    </div>
   </footer>
   <!--dibu-->
  </div>
 </body>
</html>